package org.hxzon.demo.jt.logsearcher.util;

import java.io.File;
import java.io.IOException;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.LineIterator;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.RAMDirectory;
import org.hxzon.util.DateFormatUtil;

public class LogFileIndexer {

//    private static final boolean create = true;//false;
    private static final int timeEndIndex = 15;
    private final File baseDir;
    private final String analyzerName;
    //
    private Analyzer analyzer = null;
    private int docNum = 0;
    private Directory fsIndexDirectory = null;
    private Directory ramIndexDirectory = null;
    private IndexWriter ramIndexWriter = null;
    private IndexWriter fsIndexWriter = null;

    //private 

    // --------------
    // baseDir 索引的根目录，该对象建立的所有索引均放在该目录内
    public LogFileIndexer(String baseDir, String anaName) {
        this.baseDir = new File(baseDir);
        this.analyzerName = anaName;
    }

    // 建立索引
    // indexName 索引名称，对应一个索引子目录
    // logFile 待索引的日志文件
    // 返回索引的记录数（空行不索引，也不计记录数）
    public int index(String indexName, File logFile) throws IOException {
        String indexPath = baseDir.getAbsolutePath() + "/" + indexName;
        try {
            analyzer = LogSearcherConfig.getAnalyzer(analyzerName);
            IndexWriterConfig indexWriterConfig = new IndexWriterConfig(LogSearcherConfig.LuceneVersion, analyzer);

//            if (create) {
//                indexWriterConfig.setOpenMode(OpenMode.CREATE);
//            } else {
//                indexWriterConfig.setOpenMode(OpenMode.CREATE_OR_APPEND);
//            }
            fsIndexDirectory = FSDirectory.open(new File(indexPath));
            ramIndexDirectory = new RAMDirectory();
            // indexWriterConfig.setRAMBufferSizeMB(256.0);

            ramIndexWriter = new IndexWriter(ramIndexDirectory, new IndexWriterConfig(LogSearcherConfig.LuceneVersion, analyzer));
            fsIndexWriter = new IndexWriter(fsIndexDirectory, indexWriterConfig);
            indexDocs(logFile);
            fsIndexWriter.close();
        } catch (IOException e) {
            System.out.println(" caught a " + e.getClass() + "\n with message: " + e.getMessage());
        }
        return 0;
    }

    private void indexDocs(File file) throws IOException {
        // do not try to index files that cannot be read
        if (file.canRead()) {
            if (file.isDirectory()) {
                String[] files = file.list();
                // an IO error could occur
                if (files != null) {
                    for (int i = 0; i < files.length; i++) {
                        indexDocs(new File(file, files[i]));
                    }
                }
            } else {
                indexDoc(file);
            }
        }
    }

    private void indexDoc(File file) throws IOException {
        LineIterator lit = FileUtils.lineIterator(file, LogSearcherConfig.LogFileEncoding);
        while (lit.hasNext()) {
            docNum++;
            Document doc = new Document();
            String line = lit.next();
            String logTime = line.substring(1, timeEndIndex - 2);//remove last whitespace,remove [ and ]
            String lineContent = line.substring(timeEndIndex);
            Field logTimeField = new StringField(LogSearcherConfig.FieldName_LogTime, logTime, Field.Store.YES);
            doc.add(logTimeField);

            Field contentField = new TextField(LogSearcherConfig.FieldName_Content, lineContent, Field.Store.YES);
            doc.add(contentField);
            //
            //if (ramIndexWriter.getConfig().getOpenMode() == OpenMode.CREATE) {
                // New index, so we just add the document (no old document can be there):
                ramIndexWriter.addDocument(doc);
//            } else {
//                ramIndexWriter.updateDocument(new Term(LogSearcherConfig.FieldName_LogTime, logTime), doc);//?
//            }
            if (docNum == 800000) {
                ramIndexWriter.close();
                fsIndexWriter.addIndexes(new Directory[] { ramIndexDirectory });
                //
                ramIndexWriter = new IndexWriter(ramIndexDirectory, new IndexWriterConfig(LogSearcherConfig.LuceneVersion, analyzer));
                docNum = 0;
            }
        }
    }

    public static void main(String[] args) throws IOException {
        String ana="smartcn";
        System.out.println(DateFormatUtil.formatToUniteTime());
        String baseDir = LogSearcherConfig.IndexFileBaseDir;
        String indexName = "res-"+ana+"-1";
        String logFile = "D:/work-juntai/训练题/文本全文检索应用-测试数据/res 2012-09-26.log";//res 2012-09-26//msg 2012-07-17
        LogFileIndexer indexer = new LogFileIndexer(baseDir, ana);
        indexer.index(indexName, new File(logFile));
        System.out.println(DateFormatUtil.formatToUniteTime());
    }

}
